﻿/// -----------------------------------------------------------------------
/// (c) 2009 Microsoft Corporation -- All rights reserved
/// This code is licensed under the MS-PL
/// http://www.opensource.org/licenses/ms-pl.html
/// Courtesy of the Open Source Techology Center: http://port25.technet.com
/// -----------------------------------------------------------------------

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using gsToolkit;

namespace WebWalker {
    class Program {
        static string help = @"
Usage:
-------

WebWalker [options] start-url

    options:
    --------
    --help                      this help
    --nologo                    don't display the logo
    --load-config=<configfile>  loads options from configuration file
                                List the options, one per line, without the
                                prefix '--'.
    --log-urls=<filename>       logs the URLs walked to <filename>
    --log-forms=<filename>      creates form-data lines for walked pages in <filename>

    --no-output                 does not save the files, just crawls them

    --max-requests=<number>     maximum number of requests to service

    --output=<filename>         write files to filename. Filename can contain
                                the following replaceable strings:
                                    {{filename}}  - substitutes for the original
                                                  filename, no extension
                                    {{ext}}         original extension
                                    {{folder}}    - original folder
                                    {{subfolder}} - original folder without leading /
                                    {{count}}     - a running count of the files
                                                  downloaded.
                                    {{date}}      - the current date (y-m-d)
                                    {{date-long}} - the date in long format
                                    {{time}}      - the current time (Hh:mm:ss)
                                    {{time-long}} - the current time in long fmt
                                    {{ticks}}     - the current timestamp as tics


    --url-mask=<partial-url>    subsequent URLs must match the partial-url
                                expression (regular expressions ok)

    --form-data=[""<page-mask>"",""<form-xpath>""]<urlencoded-data>
                                fills in form fields, for a specfic page,
                                and specific form:
                                    <page-mask> - an URL mask (RegEx ok)
                                    <form-xpath> - an XPATH to find the form
                                    <urlencoded-data> - the values for the
                                        form, encoded in urlencoded format.
                                        To use the default value from the input
                                        control, leave out the field name, or
                                        use a question mark (?) for the value.

                                Forms will only be submitted if there is a
                                matching form-data option.


    --cookie-data=<filename>    file to load cookie data from
    --save-cookies              save cookie data back to the file

    Notes:
        * any text intended to be filenames are cleaned to eliminate text
          characters that are filesystem unfriendly.

        ! Remember: use the caret ( ^ ) to escape any ampersands in the
                    command line (but *not* in a config-file)

    Example:
        WebWalker --form-data[""/.*/login.php"", ""//form[1]""]username=Garrett+Serack^&password=kitten http://localhost/index.php
";

        private static bool failTripped = false;

   static int Main(string[] args) {
            return new Program().main(args);
        }

        int main(string[] args) {
            Walker webWalker = new Walker();

            #region Options and Parameters
            foreach(var arg in args.Switches().Keys) {
                var parameters = args.Switches()[arg];

                switch(arg) {
                    /* options  */
                    case "log-urls":
                        webWalker.LogUrlsPath = parameters.Last();
                        break;

                    case "no-output":
                        webWalker.NoOutput = true;
                        break;

                    case "output":
                        webWalker.OutputPath = parameters.Last();
                        break;

                    case "max-requests":
                        webWalker.MaxRequests = parameters.Last().ToInt32();
                        break;

                    case "url-mask":
                        webWalker.UrlMask = parameters;
                        break;

                    case "form-data":
                        // webWalker.FormData = TransformParametersToPerUrlLists(parameters);
                        webWalker.FormData = parameters.GetComplexOptions();
                        break;

                    case "cookie-data":
                        webWalker.CookieFile = Path.GetFullPath(parameters.Last());
                        break;

                    case "save-cookies":
                        webWalker.SaveCookies = true;
                        break;

                    /* global switches */
                    case "load-config":
                        // all ready done, but don't get too picky.
                        break;

                    case "nologo":
                        webWalker.Assembly().SetLogo("");
                        break;

                    case "help":
                        Logo();
                        return Help();

                    default:
                        Logo();
                        return Fail("Unknown parameter [--{0}]", arg);
                        break;
                }
            }

            if(args.Parameters().Count == 0) {
                Logo();
                return Fail("No starting URL specified. \r\n\r\n    Use --help for command line help.");
            }
            #endregion

            Logo();
            return webWalker.Run(args.Parameters().Last());

        }

        #region fail/help/logo
        public static int Fail(string text, params object[] par) {
            failTripped = true;
            using(new ConsoleColors(ConsoleColor.Red, ConsoleColor.Black))
                Console.WriteLine("Error:{0}", text.format(par));
            return 1;
        }

        int Help() {
            using(new ConsoleColors(ConsoleColor.White, ConsoleColor.Black))
                help.Print();
            return 0;
        }

        void Logo() {
            using(new ConsoleColors(ConsoleColor.Cyan, ConsoleColor.Black))
                this.Assembly().Logo().Print();
            this.Assembly().SetLogo("");
        }
        #endregion

        #region TransformParameters
        private Dictionary<string, List<string>> TransformParametersToPerUrlLists(List<string> parameters) {
            var result = new Dictionary<string, List<string>>();
            result[ProgramExtensions.REGEX_ALL] = new List<string>();

            // GS01: doesn't support [] in regexes... if there is a problem, solve it with \] later.
            Regex rx = new Regex(@"\[(.*?)\](.*)");

            foreach(string s in parameters) {
                Match m = rx.Match(s);
                if(m.Success) {
                    if(!result.ContainsKey(m.Groups[1].Value))
                        result.Add(m.Groups[1].Value, new List<string>());
                    result[m.Groups[1].Value].Add(m.Groups[2].Value);
                }
                else {
                    result[ProgramExtensions.REGEX_ALL].Add(s);
                }
            }
            return result;
        }

        private Dictionary<string, string> TransformParametersToPerUrlDictionaries(List<string> parameters) {
            return TransformParametersToPerUrlDictionaries(parameters, null);
        }

        private Dictionary<string, string> TransformParametersToPerUrlDictionaries(List<string> parameters, string DEFAULT_Value) {
            var result = new Dictionary<string, string>();
            if(DEFAULT_Value != null)
                result[ProgramExtensions.REGEX_ALL] = DEFAULT_Value;

            // GS01: doesn't support [] in regexes... if there is a problem, solve it with \] later.
            Regex rx = new Regex(@"\[(.*?)\](.*)");

            foreach(string s in parameters) {
                Match m = rx.Match(s);
                if(m.Success) {
                    result.Add(m.Groups[1].Value, m.Groups[2].Value);
                }
                else {
                    result[ProgramExtensions.REGEX_ALL] = s;
                }
            }
            return result;
        }
        #endregion

    }

    public static class ProgramExtensions {
        public const string REGEX_ALL = ".*";

        /// <summary>
        /// Looks up a single value on a per-Url basis.
        /// </summary>
        /// <param name="dict"></param>
        /// <param name="value"></param>
        /// <returns></returns>
        public static string Lookup(this Dictionary<string, string> dict, string value) {
            if(dict == null)
                return value;

            foreach(string key in dict.Keys) {
                if(key == REGEX_ALL)
                    continue;

                if(value.Match(key).Success)
                    return dict[key];
            }
            if(dict.Keys.Contains(REGEX_ALL))
                return dict[REGEX_ALL];

            return value;
        }

        /// <summary>
        /// Looks up a list of values for on a per Url basis.
        /// </summary>
        /// <param name="dict"></param>
        /// <param name="Url"></param>
        /// <returns></returns>
        public static List<string> Lookup(this Dictionary<string, List<string>> dict, string Url) {
            List<string> result = new List<string>();
            if(dict == null)
                return result;

            foreach(string key in dict.Keys) {
                if(key == REGEX_ALL)
                    continue;
                if(key[0] == '!') {
                    if(!Url.Match(key.Substring(1)).Success)
                        result.AddRange(dict[key]);
                }
                else {
                    if(Url.Match(key).Success)
                        result.AddRange(dict[key]);
                }
            }
            if(dict.Keys.Contains(REGEX_ALL))
                result.AddRange(dict[REGEX_ALL]);

            return result;
        }

        public static List<string> LookupReverse(this Dictionary<string, List<string>> dict, string Url) {
            List<string> result = new List<string>();
            if(dict == null)
                return result;

            foreach(string key in dict.Keys) {
                if(key == REGEX_ALL)
                    continue;

                if(!Url.Match(key).Success)
                    result.AddRange(dict[key]);
            }
            return result;
        }

        internal static string[] GetUris(this List<Request> requests) {
            string[] result = new string[requests.Count];
            int pos = 0;
            foreach(Request r in requests)
                result[pos++] = r.uri.AbsoluteUri;
            return result;
        }

    }
}
